models = [
:LogisticRegression,
:FluxModel,
:FluxEnsemble,
]
opt = Flux.Descent(0.01)
generators = Dict(
:Generic=>GenericGenerator(opt = opt, decision_threshold=0.5),
:Latent=>REVISEGenerator(opt = opt),
:Generic_conservative=>GenericGenerator(opt = opt, decision_threshold=0.9),
:Gravitational=>GravitationalGenerator(opt = opt),
:ClapROAR=>ClapROARGenerator(opt = opt)
)5 Mitigation Strategies
5.1 Synthetic
max_obs = 1000
catalogue = load_synthetic(max_obs)
choices = [
:linearly_separable,
:overlapping,
:circles,
:moons,
]
data_sets = filter(p -> p[1] in choices, catalogue)experiments = set_up_experiments(data_sets,models,generators)n_evals = 5
n_rounds = 50
evaluate_every = Int(round(n_rounds/n_evals))
n_folds = 5
T = 100
using Serialization
results = run_experiments(
experiments;
save_path=output_path,evaluate_every=evaluate_every,n_rounds=n_rounds, n_folds=n_folds, T=T
)
Serialization.serialize(joinpath(output_path,"results_synthetic.jls"),results)5.2 Plots
using Serialization
results = Serialization.deserialize(joinpath(output_path,"results_synthetic.jls"))using Images
line_charts = Dict()
errorbar_charts = Dict()
for (data_name, res) in results
plt = plot(res)
Images.save(joinpath(www_path, "line_chart_$(data_name).png"), plt)
line_charts[data_name] = plt
plt = plot(res,maximum(res.output.n))
Images.save(joinpath(www_path, "errorbar_chart_$(data_name).png"), plt)
errorbar_charts[data_name] = plt
end5.2.1 Line Charts
Figure 5.1 shows the evolution of the evaluation metrics over the course of the experiment.
choices = [
:linearly_separable,
:overlapping,
:circles,
:moons,
]
img_files = readdir(www_artifact_path)[contains.(readdir(www_artifact_path),"line_chart") .&& .!contains.(readdir(www_artifact_path),"latent")]
img_files = img_files[Bool.(reduce(+, map(choice -> contains.(img_files, string(choice)), choices)))]
img_files = joinpath.(www_artifact_path,img_files)
for img in img_files
display(load(img))
end5.2.2 Error Bar Charts
Figure 5.2 shows the evaluation metrics at the end of the experiments.
choices = [
:linearly_separable,
:overlapping,
:circles,
:moons,
]
img_files = readdir(www_artifact_path)[contains.(readdir(www_artifact_path),"errorbar_chart") .&& .!contains.(readdir(www_artifact_path),"latent")]
img_files = img_files[Bool.(reduce(+, map(choice -> contains.(img_files, string(choice)), choices)))]
img_files = joinpath.(www_artifact_path,img_files)
for img in img_files
display(load(img))
end5.3 Bootstrap
n_bootstrap = 100
df = run_bootstrap(results, n_bootstrap; filename=joinpath(output_path,"bootstrap_synthetic.csv"))┌──────────┬─────────┬────────────────────┬────────────────────┬──────────────────────┬──────────────┐
│ name │ scope │ data │ model │ generator │ p_value_mean │
│ String31 │ String7 │ String31 │ String31 │ String31 │ Float64 │
├──────────┼─────────┼────────────────────┼────────────────────┼──────────────────────┼──────────────┤
│ mmd │ domain │ overlapping │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ domain │ overlapping │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd │ domain │ overlapping │ LogisticRegression │ ClapROAR │ 0.0 │
│ mmd │ domain │ overlapping │ LogisticRegression │ Gravitational │ 0.0 │
│ mmd │ domain │ overlapping │ LogisticRegression │ Latent │ 0.0 │
│ mmd │ domain │ overlapping │ FluxModel │ Generic │ 0.0 │
│ mmd │ domain │ overlapping │ FluxModel │ Generic_conservative │ 0.0 │
│ mmd │ domain │ overlapping │ FluxModel │ ClapROAR │ 0.0 │
│ mmd │ domain │ overlapping │ FluxModel │ Gravitational │ 0.0 │
│ mmd │ domain │ overlapping │ FluxModel │ Latent │ 0.0 │
│ mmd │ domain │ overlapping │ FluxEnsemble │ Generic │ 0.0 │
│ mmd │ domain │ overlapping │ FluxEnsemble │ Generic_conservative │ 0.0 │
│ mmd │ domain │ overlapping │ FluxEnsemble │ ClapROAR │ 0.0 │
│ mmd │ domain │ overlapping │ FluxEnsemble │ Gravitational │ 0.0 │
│ mmd │ domain │ overlapping │ FluxEnsemble │ Latent │ 0.0 │
│ mmd │ domain │ linearly_separable │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ domain │ linearly_separable │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd │ domain │ linearly_separable │ LogisticRegression │ ClapROAR │ 0.0 │
│ mmd │ domain │ linearly_separable │ LogisticRegression │ Gravitational │ 0.092 │
│ mmd │ domain │ linearly_separable │ LogisticRegression │ Latent │ 0.75 │
│ mmd │ domain │ linearly_separable │ FluxModel │ Generic │ 0.0 │
│ mmd │ domain │ linearly_separable │ FluxModel │ Generic_conservative │ 0.0 │
│ mmd │ domain │ linearly_separable │ FluxModel │ ClapROAR │ 0.0 │
│ mmd │ domain │ linearly_separable │ FluxModel │ Gravitational │ 0.078 │
│ mmd │ domain │ linearly_separable │ FluxModel │ Latent │ 0.742 │
│ mmd │ domain │ linearly_separable │ FluxEnsemble │ Generic │ 0.0 │
│ mmd │ domain │ linearly_separable │ FluxEnsemble │ Generic_conservative │ 0.0 │
│ mmd │ domain │ linearly_separable │ FluxEnsemble │ ClapROAR │ 0.0 │
│ mmd │ domain │ linearly_separable │ FluxEnsemble │ Gravitational │ 0.05 │
│ mmd │ domain │ linearly_separable │ FluxEnsemble │ Latent │ 0.724 │
│ mmd │ domain │ circles │ LogisticRegression │ Generic │ 1.0 │
│ mmd │ domain │ circles │ LogisticRegression │ Generic_conservative │ 1.0 │
│ mmd │ domain │ circles │ LogisticRegression │ ClapROAR │ 1.0 │
│ mmd │ domain │ circles │ LogisticRegression │ Gravitational │ 1.0 │
│ mmd │ domain │ circles │ LogisticRegression │ Latent │ 1.0 │
│ mmd │ domain │ circles │ FluxModel │ Generic │ 0.994 │
│ mmd │ domain │ circles │ FluxModel │ Generic_conservative │ 0.992 │
│ mmd │ domain │ circles │ FluxModel │ ClapROAR │ 0.992 │
│ mmd │ domain │ circles │ FluxModel │ Gravitational │ 0.998 │
│ mmd │ domain │ circles │ FluxModel │ Latent │ 1.0 │
│ mmd │ domain │ circles │ FluxEnsemble │ Generic │ 0.99 │
│ mmd │ domain │ circles │ FluxEnsemble │ Generic_conservative │ 0.996 │
│ mmd │ domain │ circles │ FluxEnsemble │ ClapROAR │ 0.984 │
│ mmd │ domain │ circles │ FluxEnsemble │ Gravitational │ 0.998 │
│ mmd │ domain │ circles │ FluxEnsemble │ Latent │ 1.0 │
│ mmd │ domain │ moons │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ domain │ moons │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd │ domain │ moons │ LogisticRegression │ ClapROAR │ 0.0 │
│ mmd │ domain │ moons │ LogisticRegression │ Gravitational │ 0.0 │
│ mmd │ domain │ moons │ LogisticRegression │ Latent │ 0.0 │
│ mmd │ domain │ moons │ FluxModel │ Generic │ 0.0 │
│ mmd │ domain │ moons │ FluxModel │ Generic_conservative │ 0.0 │
│ mmd │ domain │ moons │ FluxModel │ ClapROAR │ 0.0 │
│ mmd │ domain │ moons │ FluxModel │ Gravitational │ 0.0 │
│ mmd │ domain │ moons │ FluxModel │ Latent │ 0.0 │
│ mmd │ domain │ moons │ FluxEnsemble │ Generic │ 0.0 │
│ mmd │ domain │ moons │ FluxEnsemble │ Generic_conservative │ 0.0 │
│ mmd │ domain │ moons │ FluxEnsemble │ ClapROAR │ 0.0 │
│ mmd │ domain │ moons │ FluxEnsemble │ Gravitational │ 0.0 │
│ mmd │ domain │ moons │ FluxEnsemble │ Latent │ 0.0 │
│ mmd │ model │ overlapping │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ model │ overlapping │ LogisticRegression │ Generic_conservative │ 0.066 │
│ mmd │ model │ overlapping │ LogisticRegression │ ClapROAR │ 0.178 │
│ mmd │ model │ overlapping │ LogisticRegression │ Gravitational │ 0.156 │
│ mmd │ model │ overlapping │ LogisticRegression │ Latent │ 0.006 │
│ mmd │ model │ overlapping │ FluxModel │ Generic │ 0.004 │
│ mmd │ model │ overlapping │ FluxModel │ Generic_conservative │ 0.13 │
│ mmd │ model │ overlapping │ FluxModel │ ClapROAR │ 0.486 │
│ mmd │ model │ overlapping │ FluxModel │ Gravitational │ 0.564 │
│ mmd │ model │ overlapping │ FluxModel │ Latent │ 0.046 │
│ mmd │ model │ overlapping │ FluxEnsemble │ Generic │ 0.0 │
│ mmd │ model │ overlapping │ FluxEnsemble │ Generic_conservative │ 0.122 │
│ mmd │ model │ overlapping │ FluxEnsemble │ ClapROAR │ 0.46 │
│ mmd │ model │ overlapping │ FluxEnsemble │ Gravitational │ 0.514 │
│ mmd │ model │ overlapping │ FluxEnsemble │ Latent │ 0.048 │
│ mmd │ model │ linearly_separable │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ model │ linearly_separable │ LogisticRegression │ Generic_conservative │ 0.376 │
│ mmd │ model │ linearly_separable │ LogisticRegression │ ClapROAR │ 0.916 │
│ mmd │ model │ linearly_separable │ LogisticRegression │ Gravitational │ 0.904 │
│ mmd │ model │ linearly_separable │ LogisticRegression │ Latent │ 0.428 │
│ mmd │ model │ linearly_separable │ FluxModel │ Generic │ 0.916 │
│ mmd │ model │ linearly_separable │ FluxModel │ Generic_conservative │ 0.968 │
│ mmd │ model │ linearly_separable │ FluxModel │ ClapROAR │ 0.958 │
│ mmd │ model │ linearly_separable │ FluxModel │ Gravitational │ 0.982 │
│ mmd │ model │ linearly_separable │ FluxModel │ Latent │ 0.83 │
│ mmd │ model │ linearly_separable │ FluxEnsemble │ Generic │ 0.922 │
│ mmd │ model │ linearly_separable │ FluxEnsemble │ Generic_conservative │ 0.968 │
│ mmd │ model │ linearly_separable │ FluxEnsemble │ ClapROAR │ 0.962 │
│ mmd │ model │ linearly_separable │ FluxEnsemble │ Gravitational │ 0.976 │
│ mmd │ model │ linearly_separable │ FluxEnsemble │ Latent │ 0.862 │
│ mmd │ model │ circles │ LogisticRegression │ Generic │ 0.8 │
│ mmd │ model │ circles │ LogisticRegression │ Generic_conservative │ 0.996 │
│ mmd │ model │ circles │ LogisticRegression │ ClapROAR │ 0.996 │
│ mmd │ model │ circles │ LogisticRegression │ Gravitational │ 0.0 │
│ mmd │ model │ circles │ LogisticRegression │ Latent │ 0.0 │
│ mmd │ model │ circles │ FluxModel │ Generic │ 1.0 │
│ mmd │ model │ circles │ FluxModel │ Generic_conservative │ 1.0 │
│ mmd │ model │ circles │ FluxModel │ ClapROAR │ 0.998 │
│ mmd │ model │ circles │ FluxModel │ Gravitational │ 0.986 │
│ mmd │ model │ circles │ FluxModel │ Latent │ 0.998 │
│ mmd │ model │ circles │ FluxEnsemble │ Generic │ 0.998 │
│ mmd │ model │ circles │ FluxEnsemble │ Generic_conservative │ 0.998 │
│ mmd │ model │ circles │ FluxEnsemble │ ClapROAR │ 0.998 │
│ mmd │ model │ circles │ FluxEnsemble │ Gravitational │ 0.978 │
│ mmd │ model │ circles │ FluxEnsemble │ Latent │ 1.0 │
│ mmd │ model │ moons │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ model │ moons │ LogisticRegression │ Generic_conservative │ 0.086 │
│ mmd │ model │ moons │ LogisticRegression │ ClapROAR │ 0.462 │
│ mmd │ model │ moons │ LogisticRegression │ Gravitational │ 0.446 │
│ mmd │ model │ moons │ LogisticRegression │ Latent │ 0.008 │
│ mmd │ model │ moons │ FluxModel │ Generic │ 0.812 │
│ mmd │ model │ moons │ FluxModel │ Generic_conservative │ 0.87 │
│ mmd │ model │ moons │ FluxModel │ ClapROAR │ 0.956 │
│ mmd │ model │ moons │ FluxModel │ Gravitational │ 0.984 │
│ mmd │ model │ moons │ FluxModel │ Latent │ 0.94 │
│ mmd │ model │ moons │ FluxEnsemble │ Generic │ 0.822 │
│ mmd │ model │ moons │ FluxEnsemble │ Generic_conservative │ 0.818 │
│ mmd │ model │ moons │ FluxEnsemble │ ClapROAR │ 0.966 │
│ mmd │ model │ moons │ FluxEnsemble │ Gravitational │ 0.9775 │
│ mmd │ model │ moons │ FluxEnsemble │ Latent │ 0.922 │
│ mmd_grid │ model │ overlapping │ LogisticRegression │ Generic │ 0.0 │
│ mmd_grid │ model │ overlapping │ LogisticRegression │ Generic_conservative │ 0.104 │
│ mmd_grid │ model │ overlapping │ LogisticRegression │ ClapROAR │ 0.256 │
│ mmd_grid │ model │ overlapping │ LogisticRegression │ Gravitational │ 0.27 │
│ mmd_grid │ model │ overlapping │ LogisticRegression │ Latent │ 0.01 │
│ mmd_grid │ model │ overlapping │ FluxModel │ Generic │ 0.0 │
│ mmd_grid │ model │ overlapping │ FluxModel │ Generic_conservative │ 0.116 │
│ mmd_grid │ model │ overlapping │ FluxModel │ ClapROAR │ 0.298 │
│ mmd_grid │ model │ overlapping │ FluxModel │ Gravitational │ 0.344 │
│ mmd_grid │ model │ overlapping │ FluxModel │ Latent │ 0.204 │
│ mmd_grid │ model │ overlapping │ FluxEnsemble │ Generic │ 0.002 │
│ mmd_grid │ model │ overlapping │ FluxEnsemble │ Generic_conservative │ 0.154 │
│ mmd_grid │ model │ overlapping │ FluxEnsemble │ ClapROAR │ 0.4075 │
│ mmd_grid │ model │ overlapping │ FluxEnsemble │ Gravitational │ 0.356 │
│ mmd_grid │ model │ overlapping │ FluxEnsemble │ Latent │ 0.324 │
│ mmd_grid │ model │ linearly_separable │ LogisticRegression │ Generic │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ LogisticRegression │ ClapROAR │ 0.006 │
│ mmd_grid │ model │ linearly_separable │ LogisticRegression │ Gravitational │ 0.342 │
│ mmd_grid │ model │ linearly_separable │ LogisticRegression │ Latent │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ FluxModel │ Generic │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ FluxModel │ Generic_conservative │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ FluxModel │ ClapROAR │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ FluxModel │ Gravitational │ 0.668 │
│ mmd_grid │ model │ linearly_separable │ FluxModel │ Latent │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ FluxEnsemble │ Generic │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ FluxEnsemble │ Generic_conservative │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ FluxEnsemble │ ClapROAR │ 0.0 │
│ mmd_grid │ model │ linearly_separable │ FluxEnsemble │ Gravitational │ 0.408 │
│ mmd_grid │ model │ linearly_separable │ FluxEnsemble │ Latent │ 0.0 │
│ mmd_grid │ model │ circles │ LogisticRegression │ Generic │ 0.814 │
│ mmd_grid │ model │ circles │ LogisticRegression │ Generic_conservative │ 0.988 │
│ mmd_grid │ model │ circles │ LogisticRegression │ ClapROAR │ 0.996 │
│ mmd_grid │ model │ circles │ LogisticRegression │ Gravitational │ 0.0 │
│ mmd_grid │ model │ circles │ LogisticRegression │ Latent │ 0.0 │
│ mmd_grid │ model │ circles │ FluxModel │ Generic │ 0.814 │
│ mmd_grid │ model │ circles │ FluxModel │ Generic_conservative │ 0.932 │
│ mmd_grid │ model │ circles │ FluxModel │ ClapROAR │ 0.99 │
│ mmd_grid │ model │ circles │ FluxModel │ Gravitational │ 0.406 │
│ mmd_grid │ model │ circles │ FluxModel │ Latent │ 0.99 │
│ mmd_grid │ model │ circles │ FluxEnsemble │ Generic │ 0.886 │
│ mmd_grid │ model │ circles │ FluxEnsemble │ Generic_conservative │ 0.84 │
│ mmd_grid │ model │ circles │ FluxEnsemble │ ClapROAR │ 0.984 │
│ mmd_grid │ model │ circles │ FluxEnsemble │ Gravitational │ 0.55 │
│ mmd_grid │ model │ circles │ FluxEnsemble │ Latent │ 0.996 │
│ mmd_grid │ model │ moons │ LogisticRegression │ Generic │ 0.0 │
│ mmd_grid │ model │ moons │ LogisticRegression │ Generic_conservative │ 0.09 │
│ mmd_grid │ model │ moons │ LogisticRegression │ ClapROAR │ 0.458 │
│ mmd_grid │ model │ moons │ LogisticRegression │ Gravitational │ 0.456 │
│ mmd_grid │ model │ moons │ LogisticRegression │ Latent │ 0.008 │
│ mmd_grid │ model │ moons │ FluxModel │ Generic │ 0.016 │
│ mmd_grid │ model │ moons │ FluxModel │ Generic_conservative │ 0.03 │
│ mmd_grid │ model │ moons │ FluxModel │ ClapROAR │ 0.004 │
│ mmd_grid │ model │ moons │ FluxModel │ Gravitational │ 0.426 │
│ mmd_grid │ model │ moons │ FluxModel │ Latent │ 0.114 │
│ mmd_grid │ model │ moons │ FluxEnsemble │ Generic │ 0.0 │
│ mmd_grid │ model │ moons │ FluxEnsemble │ Generic_conservative │ 0.006 │
│ mmd_grid │ model │ moons │ FluxEnsemble │ ClapROAR │ 0.0 │
│ mmd_grid │ model │ moons │ FluxEnsemble │ Gravitational │ 0.4 │
│ mmd_grid │ model │ moons │ FluxEnsemble │ Latent │ 0.344 │
└──────────┴─────────┴────────────────────┴────────────────────┴──────────────────────┴──────────────┘
5.4 Chart in paper
Figure 5.3 shows the chart that went into the paper.
Images.load(joinpath(www_artifact_path,"paper_synthetic_results.png"))5.5 Latent Space Search
generators = Dict(
:Latent=>GenericGenerator(opt = opt, decision_threshold=0.5),
:Latent_conservative=>GenericGenerator(opt = opt, decision_threshold=0.9),
:Gravitational=>GravitationalGenerator(opt = opt),
:ClapROAR=>ClapROARGenerator(opt = opt)
)experiments = set_up_experiments(data_sets,models,generators)n_evals = 5
n_rounds = 50
evaluate_every = Int(round(n_rounds/n_evals))
n_folds = 5
T = 100
using Serialization
results = run_experiments(
experiments;
save_path=output_path, evaluate_every=evaluate_every, n_rounds=n_rounds, n_folds=n_folds, T=T,
latent_space = true
)
Serialization.serialize(joinpath(output_path,"results_synthetic_latent.jls"),results)using Serialization
results = Serialization.deserialize(joinpath(output_path,"results_synthetic_latent.jls"))using Images
line_charts = Dict()
errorbar_charts = Dict()
for (data_name, res) in results
plt = plot(res)
Images.save(joinpath(www_path, "line_chart_latent_$(data_name).png"), plt)
line_charts[data_name] = plt
plt = plot(res,maximum(res.output.n))
Images.save(joinpath(www_path, "errorbar_chart_latent_$(data_name).png"), plt)
errorbar_charts[data_name] = plt
end5.6 Plots
5.6.1 Line Charts
Figure 5.4 shows the evolution of the evaluation metrics over the course of the experiment.
img_files = readdir(www_artifact_path)[contains.(readdir(www_artifact_path),"line_chart") .&& contains.(readdir(www_artifact_path),"latent")]
img_files = joinpath.(www_artifact_path,img_files)
for img in img_files
display(load(img))
end5.6.2 Error Bar Charts
Figure 5.5 shows the evaluation metrics at the end of the experiments.
img_files = readdir(www_artifact_path)[contains.(readdir(www_artifact_path),"errorbar_chart") .&& contains.(readdir(www_artifact_path),"latent")]
img_files = joinpath.(www_artifact_path,img_files)
for img in img_files
display(load(img))
end5.7 Bootstrap
n_bootstrap = 100
df = run_bootstrap(results, n_bootstrap; filename=joinpath(output_path,"bootstrap_latent.csv"))5.8 Chart in paper
Figure 5.6 shows the chart that went into the paper.
Images.load(joinpath(www_artifact_path,"paper_synthetic_latent_results.png"))5.9 Real World
models = [
:LogisticRegression,
:FluxModel,
:FluxEnsemble,
]
opt = Flux.Descent(0.01)
generators = Dict(
:Generic=>GenericGenerator(opt = opt, decision_threshold=0.5),
:Latent=>REVISEGenerator(opt = opt),
:Generic_conservative=>GenericGenerator(opt = opt, decision_threshold=0.9),
:Gravitational=>GravitationalGenerator(opt = opt),
:ClapROAR=>ClapROARGenerator(opt = opt)
)max_obs = 2500
data_path = data_dir("real_world")
data_sets = load_real_world(max_obs)
choices = [
:cal_housing,
:credit_default,
:gmsc,
]
data_sets = filter(p -> p[1] in choices, data_sets)using CounterfactualExplanations.DataPreprocessing: unpack
bs = 500
function data_loader(data::CounterfactualData)
X, y = unpack(data)
data = Flux.DataLoader((X,y),batchsize=bs)
return data
end
model_params = (batch_norm=false,n_hidden=64,n_layers=3,dropout=true,p_dropout=0.1)experiments = set_up_experiments(
data_sets,models,generators;
pre_train_models=100, model_params=model_params,
data_loader=data_loader
)n_evals = 5
n_rounds = 50
evaluate_every = Int(round(n_rounds/n_evals))
n_folds = 5
n_samples = 10000
T = 100
generative_model_params = (epochs=250, latent_dim=8)
results = run_experiments(
experiments;
save_path=output_path,evaluate_every=evaluate_every,n_rounds=n_rounds, n_folds=n_folds, T=T, n_samples=n_samples,
generative_model_params=generative_model_params
)
Serialization.serialize(joinpath(output_path,"results_real_world.jls"),results)using Serialization
results = Serialization.deserialize(joinpath(output_path,"results_real_world.jls"))using Images
line_charts = Dict()
errorbar_charts = Dict()
for (data_name, res) in results
plt = plot(res)
Images.save(joinpath(www_path, "line_chart_$(data_name).png"), plt)
line_charts[data_name] = plt
plt = plot(res,maximum(res.output.n))
Images.save(joinpath(www_path, "errorbar_chart_$(data_name).png"), plt)
errorbar_charts[data_name] = plt
end5.9.1 Line Charts
Figure 4.1 shows the evolution of the evaluation metrics over the course of the experiment.
choices = [
:cal_housing,
:credit_default,
:gmsc,
]
img_files = readdir(www_artifact_path)[contains.(readdir(www_artifact_path),"line_chart")]
img_files = img_files[Bool.(reduce(+, map(choice -> contains.(img_files, string(choice)), choices)))]
img_files = joinpath.(www_artifact_path,img_files)
for img in img_files
display(load(img))
end5.9.2 Error Bar Charts
Figure 4.2 shows the evaluation metrics at the end of the experiments.
choices = [
:cal_housing,
:credit_default,
:gmsc,
]
img_files = readdir(www_artifact_path)[contains.(readdir(www_artifact_path),"errorbar_chart")]
img_files = img_files[Bool.(reduce(+, map(choice -> contains.(img_files, string(choice)), choices)))]
img_files = joinpath.(www_artifact_path,img_files)
for img in img_files
display(load(img))
end5.9.3 Bootstrap
n_bootstrap = 100
df = run_bootstrap(results, n_bootstrap; filename=joinpath(output_path,"bootstrap_real_world.csv"))┌──────────┬─────────┬────────────────┬────────────────────┬──────────────────────┬──────────────┐
│ name │ scope │ data │ model │ generator │ p_value_mean │
│ String31 │ String7 │ String15 │ String31 │ String31 │ Float64 │
├──────────┼─────────┼────────────────┼────────────────────┼──────────────────────┼──────────────┤
│ mmd │ domain │ credit_default │ FluxModel │ Generic_conservative │ 1.0 │
│ mmd │ domain │ credit_default │ FluxModel │ Generic │ 1.0 │
│ mmd │ domain │ credit_default │ FluxModel │ ClapROAR │ 1.0 │
│ mmd │ domain │ credit_default │ FluxModel │ Gravitational │ 0.0 │
│ mmd │ domain │ credit_default │ FluxModel │ Latent │ 0.0 │
│ mmd │ domain │ credit_default │ LogisticRegression │ Generic_conservative │ 1.0 │
│ mmd │ domain │ credit_default │ LogisticRegression │ Generic │ 1.0 │
│ mmd │ domain │ credit_default │ LogisticRegression │ ClapROAR │ 1.0 │
│ mmd │ domain │ credit_default │ LogisticRegression │ Gravitational │ 0.0 │
│ mmd │ domain │ credit_default │ LogisticRegression │ Latent │ 0.8 │
│ mmd │ domain │ credit_default │ FluxEnsemble │ Generic_conservative │ 1.0 │
│ mmd │ domain │ credit_default │ FluxEnsemble │ Generic │ 1.0 │
│ mmd │ domain │ credit_default │ FluxEnsemble │ ClapROAR │ 1.0 │
│ mmd │ domain │ credit_default │ FluxEnsemble │ Gravitational │ 0.0 │
│ mmd │ domain │ credit_default │ FluxEnsemble │ Latent │ 0.0 │
│ mmd │ domain │ cal_housing │ FluxModel │ Generic_conservative │ 0.0 │
│ mmd │ domain │ cal_housing │ FluxModel │ Generic │ 0.0 │
│ mmd │ domain │ cal_housing │ FluxModel │ ClapROAR │ 0.0 │
│ mmd │ domain │ cal_housing │ FluxModel │ Gravitational │ 0.0 │
│ mmd │ domain │ cal_housing │ FluxModel │ Latent │ 0.0 │
│ mmd │ domain │ cal_housing │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd │ domain │ cal_housing │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ domain │ cal_housing │ LogisticRegression │ ClapROAR │ 0.0 │
│ mmd │ domain │ cal_housing │ LogisticRegression │ Gravitational │ 0.0 │
│ mmd │ domain │ cal_housing │ LogisticRegression │ Latent │ 0.0 │
│ mmd │ domain │ cal_housing │ FluxEnsemble │ Generic_conservative │ 0.0 │
│ mmd │ domain │ cal_housing │ FluxEnsemble │ Generic │ 0.0 │
│ mmd │ domain │ cal_housing │ FluxEnsemble │ ClapROAR │ 0.0 │
│ mmd │ domain │ cal_housing │ FluxEnsemble │ Gravitational │ 0.0 │
│ mmd │ domain │ cal_housing │ FluxEnsemble │ Latent │ 0.0 │
│ mmd │ domain │ gmsc │ FluxModel │ Generic_conservative │ 0.93 │
│ mmd │ domain │ gmsc │ FluxModel │ Generic │ 0.932 │
│ mmd │ domain │ gmsc │ FluxModel │ ClapROAR │ 0.214 │
│ mmd │ domain │ gmsc │ FluxModel │ Gravitational │ 0.0 │
│ mmd │ domain │ gmsc │ FluxModel │ Latent │ 0.0 │
│ mmd │ domain │ gmsc │ LogisticRegression │ Generic_conservative │ 0.004 │
│ mmd │ domain │ gmsc │ LogisticRegression │ Generic │ 0.856 │
│ mmd │ domain │ gmsc │ LogisticRegression │ ClapROAR │ 0.0 │
│ mmd │ domain │ gmsc │ LogisticRegression │ Gravitational │ 0.0 │
│ mmd │ domain │ gmsc │ LogisticRegression │ Latent │ 0.0 │
│ mmd │ domain │ gmsc │ FluxEnsemble │ Generic_conservative │ 0.758 │
│ mmd │ domain │ gmsc │ FluxEnsemble │ Generic │ 0.938 │
│ mmd │ domain │ gmsc │ FluxEnsemble │ ClapROAR │ 0.15 │
│ mmd │ domain │ gmsc │ FluxEnsemble │ Gravitational │ 0.0 │
│ mmd │ domain │ gmsc │ FluxEnsemble │ Latent │ 0.0 │
│ mmd │ model │ credit_default │ FluxModel │ Generic_conservative │ 0.0 │
│ mmd │ model │ credit_default │ FluxModel │ Generic │ 0.0 │
│ mmd │ model │ credit_default │ FluxModel │ ClapROAR │ 0.0 │
│ mmd │ model │ credit_default │ FluxModel │ Gravitational │ 0.0 │
│ mmd │ model │ credit_default │ FluxModel │ Latent │ 0.0 │
│ mmd │ model │ credit_default │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd │ model │ credit_default │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ model │ credit_default │ LogisticRegression │ ClapROAR │ 0.0 │
│ mmd │ model │ credit_default │ LogisticRegression │ Gravitational │ 0.0 │
│ mmd │ model │ credit_default │ LogisticRegression │ Latent │ 0.0 │
│ mmd │ model │ credit_default │ FluxEnsemble │ Generic_conservative │ 0.0 │
│ mmd │ model │ credit_default │ FluxEnsemble │ Generic │ 0.0 │
│ mmd │ model │ credit_default │ FluxEnsemble │ ClapROAR │ 0.0 │
│ mmd │ model │ credit_default │ FluxEnsemble │ Gravitational │ 0.0 │
│ mmd │ model │ credit_default │ FluxEnsemble │ Latent │ 0.0 │
│ mmd │ model │ cal_housing │ FluxModel │ Generic_conservative │ 0.0 │
│ mmd │ model │ cal_housing │ FluxModel │ Generic │ 0.0 │
│ mmd │ model │ cal_housing │ FluxModel │ ClapROAR │ 0.0 │
│ mmd │ model │ cal_housing │ FluxModel │ Gravitational │ 0.0 │
│ mmd │ model │ cal_housing │ FluxModel │ Latent │ 0.0 │
│ mmd │ model │ cal_housing │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd │ model │ cal_housing │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ model │ cal_housing │ LogisticRegression │ ClapROAR │ 0.0 │
│ mmd │ model │ cal_housing │ LogisticRegression │ Gravitational │ 0.0 │
│ mmd │ model │ cal_housing │ LogisticRegression │ Latent │ 0.0 │
│ mmd │ model │ cal_housing │ FluxEnsemble │ Generic_conservative │ 0.0 │
│ mmd │ model │ cal_housing │ FluxEnsemble │ Generic │ 0.0 │
│ mmd │ model │ cal_housing │ FluxEnsemble │ ClapROAR │ 0.0 │
│ mmd │ model │ cal_housing │ FluxEnsemble │ Gravitational │ 0.0 │
│ mmd │ model │ cal_housing │ FluxEnsemble │ Latent │ 0.0 │
│ mmd │ model │ gmsc │ FluxModel │ Generic_conservative │ 0.0 │
│ mmd │ model │ gmsc │ FluxModel │ Generic │ 0.0 │
│ mmd │ model │ gmsc │ FluxModel │ ClapROAR │ 0.0 │
│ mmd │ model │ gmsc │ FluxModel │ Gravitational │ 0.0 │
│ mmd │ model │ gmsc │ FluxModel │ Latent │ 0.0 │
│ mmd │ model │ gmsc │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd │ model │ gmsc │ LogisticRegression │ Generic │ 0.0 │
│ mmd │ model │ gmsc │ LogisticRegression │ ClapROAR │ 0.0 │
│ mmd │ model │ gmsc │ LogisticRegression │ Gravitational │ 0.0 │
│ mmd │ model │ gmsc │ LogisticRegression │ Latent │ 0.0 │
│ mmd │ model │ gmsc │ FluxEnsemble │ Generic_conservative │ 0.0 │
│ mmd │ model │ gmsc │ FluxEnsemble │ Generic │ 0.0 │
│ mmd │ model │ gmsc │ FluxEnsemble │ ClapROAR │ 0.0 │
│ mmd │ model │ gmsc │ FluxEnsemble │ Gravitational │ 0.0 │
│ mmd │ model │ gmsc │ FluxEnsemble │ Latent │ 0.0 │
│ mmd_grid │ model │ credit_default │ FluxModel │ Generic_conservative │ 0.0 │
│ mmd_grid │ model │ credit_default │ FluxModel │ Generic │ 0.0 │
│ mmd_grid │ model │ credit_default │ FluxModel │ ClapROAR │ 0.0 │
│ mmd_grid │ model │ credit_default │ FluxModel │ Gravitational │ 0.0 │
│ mmd_grid │ model │ credit_default │ FluxModel │ Latent │ 0.0 │
│ mmd_grid │ model │ credit_default │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd_grid │ model │ credit_default │ LogisticRegression │ Generic │ 0.0 │
│ mmd_grid │ model │ credit_default │ LogisticRegression │ ClapROAR │ 0.0 │
│ mmd_grid │ model │ credit_default │ LogisticRegression │ Gravitational │ 0.0 │
│ mmd_grid │ model │ credit_default │ LogisticRegression │ Latent │ 0.078 │
│ mmd_grid │ model │ credit_default │ FluxEnsemble │ Generic_conservative │ 0.0 │
│ mmd_grid │ model │ credit_default │ FluxEnsemble │ Generic │ 0.0 │
│ mmd_grid │ model │ credit_default │ FluxEnsemble │ ClapROAR │ 0.0 │
│ mmd_grid │ model │ credit_default │ FluxEnsemble │ Gravitational │ 0.0 │
│ mmd_grid │ model │ credit_default │ FluxEnsemble │ Latent │ 0.0 │
│ mmd_grid │ model │ cal_housing │ FluxModel │ Generic_conservative │ 0.0 │
│ mmd_grid │ model │ cal_housing │ FluxModel │ Generic │ 0.0 │
│ mmd_grid │ model │ cal_housing │ FluxModel │ ClapROAR │ 0.012 │
│ mmd_grid │ model │ cal_housing │ FluxModel │ Gravitational │ 0.0625 │
│ mmd_grid │ model │ cal_housing │ FluxModel │ Latent │ 0.0 │
│ mmd_grid │ model │ cal_housing │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd_grid │ model │ cal_housing │ LogisticRegression │ Generic │ 0.0 │
│ mmd_grid │ model │ cal_housing │ LogisticRegression │ ClapROAR │ 0.004 │
│ mmd_grid │ model │ cal_housing │ LogisticRegression │ Gravitational │ 0.014 │
│ mmd_grid │ model │ cal_housing │ LogisticRegression │ Latent │ 0.002 │
│ mmd_grid │ model │ cal_housing │ FluxEnsemble │ Generic_conservative │ 0.002 │
│ mmd_grid │ model │ cal_housing │ FluxEnsemble │ Generic │ 0.0 │
│ mmd_grid │ model │ cal_housing │ FluxEnsemble │ ClapROAR │ 0.044 │
│ mmd_grid │ model │ cal_housing │ FluxEnsemble │ Gravitational │ 0.0 │
│ mmd_grid │ model │ cal_housing │ FluxEnsemble │ Latent │ 0.0 │
│ mmd_grid │ model │ gmsc │ FluxModel │ Generic_conservative │ 0.0 │
│ mmd_grid │ model │ gmsc │ FluxModel │ Generic │ 0.0 │
│ mmd_grid │ model │ gmsc │ FluxModel │ ClapROAR │ 0.0 │
│ mmd_grid │ model │ gmsc │ FluxModel │ Gravitational │ 0.0 │
│ mmd_grid │ model │ gmsc │ FluxModel │ Latent │ 0.0 │
│ mmd_grid │ model │ gmsc │ LogisticRegression │ Generic_conservative │ 0.0 │
│ mmd_grid │ model │ gmsc │ LogisticRegression │ Generic │ 0.0 │
│ mmd_grid │ model │ gmsc │ LogisticRegression │ ClapROAR │ 0.0 │
│ mmd_grid │ model │ gmsc │ LogisticRegression │ Gravitational │ 0.0 │
│ mmd_grid │ model │ gmsc │ LogisticRegression │ Latent │ 0.0 │
│ mmd_grid │ model │ gmsc │ FluxEnsemble │ Generic_conservative │ 0.0 │
│ mmd_grid │ model │ gmsc │ FluxEnsemble │ Generic │ 0.0 │
│ mmd_grid │ model │ gmsc │ FluxEnsemble │ ClapROAR │ 0.0 │
│ mmd_grid │ model │ gmsc │ FluxEnsemble │ Gravitational │ 0.0 │
│ mmd_grid │ model │ gmsc │ FluxEnsemble │ Latent │ 0.0 │
└──────────┴─────────┴────────────────┴────────────────────┴──────────────────────┴──────────────┘
5.9.4 Chart in paper
Figure 5.9 shows the chart that went into the paper.
Images.load(joinpath(www_artifact_path,"paper_real_world_results.png"))